package fetcher

import (
	"bufio"
	"crawler_distributied/config"
	"fmt"
	"io"
	"log"
	"math/rand"
	"net/http"
	"time"

	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
)

// 时间间隔限制
var rateLimiter = time.Tick(time.Second / config.Qps)

// 获取数据 输入url 输出结果
func Fetch(url string) ([]byte, error) {
	<-rateLimiter
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}

	// 设置header
	req.Header.Set(
		"User-Agent",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.86")
	req.Header.Set("Cookie", "Hm_lvt_2c8ad67df9e787ad29dbd54ee608f5d2=1708221594,1708227341; FSSBBIl1UgzbN7NO=5gR.CaqpkjkvCvyQ8QI8lCfuCJKxy9GcwSQOyfWdKg.9sYQbaErBnGrHV7Yuin5uk0uenccnS32AIBJGUfu.ADa; ec=tkqZ9dKo-1708241235981-24234341896dd184659895; sid=uMJ2y5t1oWMisnmVg2bv; token=1479482888.1708566219288.47b7deaa14a19c58aaab53ceec5314d0; refreshToken=1479482888.1708652619288.79784cc00fa436f5a11c98820e70f761; recommendId=ZaMainpageTa-0001-online_xgb-0.0.1-za_user_profile-0.0.1; lrt=1708671856341; Hm_lpvt_2c8ad67df9e787ad29dbd54ee608f5d2=1708671918; _exid=vgyTngoidwzG6nTcrV%2FX3y%2F4pI23llvlskr2LNmxr5u1MAWVYdjNhbMC1pZiT8lNHOBXDogERyc%2BWi0LWlSWIg%3D%3D; _efmdata=s57vU%2BXiHSNYzbSRD1%2FB1Z77%2FRqXOHmKt%2BYMSZLUUvWSiwx64xfioLjXiOmeQw7%2F1vtYJbrFjY9oLngR%2Bud269o%2BdnwV%2FwHG%2Bd%2B6mJMy28g%3D; FSSBBIl1UgzbN7NP=5RwFvpKUKNuQqqqDJTY7gwqcACih23kzhgeOCOlN_Y22C9EDKQZ5n7ivRBaQ1ulYKvXVKN9hRSznrWo.Fxl.2C52dYhqYDsN3n0Z1Gqp2Gv6ESPl7DtOzeTH8z27UZDrdQleFACWtHrio3xVaKy3U26JjoGQCIqV5otya7XNdYrHjD5HhkctN91JgNPCphmsyvvN3ddiqbcPgja6tWs536gXpFa12V.BIBD958O8XBpdQb1e__jp0omgG1cyM1BHBgwvpsC5dp6U_Y4Ygi8yGe1hQLp7ea9ZThn1h.lV7wfikTYAkAE70SubTmVWZqHCKZ")
	//req.Header.Set("User-Agent", getRandomAgent())
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	// 错误处理
	if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusAccepted) {
		return nil, fmt.Errorf("wrong status code: %d", resp.StatusCode)
	}

	// 把网页转为utf-8编码
	bodyReader := bufio.NewReader(resp.Body)
	e := determineEncoding(bodyReader)                           // 获取html的编码方式
	utf8Reader := transform.NewReader(resp.Body, e.NewDecoder()) // 按照html的编码方式 解析html为 utf-8 编码
	return io.ReadAll(utf8Reader)
}

// 获取html的编码方式
func determineEncoding(r *bufio.Reader) encoding.Encoding {
	bytes, err := r.Peek(1024)
	if err != nil {
		log.Printf("Fetcher error: %v", err)
		return unicode.UTF8
	}

	e, _, _ := charset.DetermineEncoding(bytes, "")
	return e
}

// 随机请求头
func getRandomAgent() string {
	agent := [...]string{
		"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0",
		"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
		"Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
		"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
		"Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
		"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
		"User-Agent,Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
		"User-Agent, Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
		"User-Agent,Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
	}

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	len := len(agent)
	return agent[r.Intn(len)]
}
